# The purpose of this program is to compute the query terms of which the freq is great or equal to 20 in the training query set
# step2-method2: compute the freq of freq for this distribution
# Updated by Wei 2013/04/19
# CURRENT applied method2.
print "do step2"

freqOfFreqDict = {}
freqOfFreqWithQueryTermsList = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%_sortedByQueryTermFreq"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/freqOfFreqInQueries_head_85K_0_85%_withQueryTemrs_sortedByFreqR"
outputFileHandler = open(outputFileName,"w")

for i in range(1,11473):
    freqOfFreqDict[i] = 0
    freqOfFreqWithQueryTermsList[i] = []

print "len(freqOfFreqDict):",len(freqOfFreqDict)

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freq = int(lineElements[1])
    if freq >= 20:
        freqOfFreqDict[freq] += 1
        freqOfFreqWithQueryTermsList[freq].append(queryTerm)
    else:
        # for the query term which has the freq < 20
        # we just don't care in this case
        pass

# output the freq of freq
for i in range(1,11473):
    outputLine = str(i) + " " + str( freqOfFreqDict[i] ) + " "
    freqOfFreqWithQueryTermsList[i].sort()
    
    for queryTerm in freqOfFreqWithQueryTermsList[i]:
        outputLine += queryTerm + " "
    
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine)

inputFileHandler.close()
outputFileHandler.close()
